/*
 * Test.java
 *
 * Created on October 21, 2002, 11:16 AM
 */

package com.xtera.nuwave.tools.redundancy.test;

import java.net.*;
import java.io.*;
import java.util.*;
import javax.net.*;
import com.sun.net.ssl.*;
import javax.net.ssl.*;
import org.apache.log4j.*;

/**
 *
 * @author  llaranje
 * @version 
 */
public class Test {
    static boolean stop = false;
    static boolean fromstart = true;
    static int firsttest = 1;
    static PrintWriter out1, out2;
    static BufferedReader in1, in2, testin;

    static String appname = "EMS";
    static final String ruleofthumb = "4c6c14a5de1cacbdab5dcf71e0078c7f";
    static SSLSocketFactory sslFact;
    
    private static final String RMCFGFILENAME = "RedMan.conf";
    private static final String EMSSTACFGFILENAME = "EMSstatic.conf";
    private static final String EMSDYNACFGFILENAME = "EMSdynamic.conf";
    private static final String TESTCFGFILENAME = "Test.conf";
    
    static String rmcfgfilepath;
    static String rmcfgfiledir;
    static String emsstacfgfilepath;
    static String emsdynacfgfilepath;
    static String testcfgfilepath = TESTCFGFILENAME;
    static String emscfgfiledir;
    static FileInputStream rmcfgfile;
    static FileInputStream emsstacfgfile;
    static FileInputStream emsdynacfgfile;

    
    private static Log4Test logstart;
    private static Category trace;
    
    private static Properties rmcfgtable;
    private static Properties emsstacfgtable;
    private static Properties emsdynacfgtable;
    private static Properties testcfgtable;
    
    static String apphosts;
    static String host1, host2;
    static String primaryhost;
    static String localhost;
    static String remotehost;
    static String appconfig;
    static String[] procnames;
    static String[] procpids;
    static String equality;
    static boolean failoverenabled;
    static boolean startenabled;   
    static int shutdowntime;
    static int inittime;
    static int statechgtime;
    static int numproc;
    
    private static int rmport;


    /** Creates new Test */
    public Test() {
    }
    
    private static void initSSL() {
        // Initialize the provider
        System.setProperty("java.protocol.handler.pkgs","com.sun.net.ssl.internal.www.protocol");
        java.security.Security.addProvider(new com.sun.net.ssl.internal.ssl.Provider()); 
            
        // Get socket factory
        sslFact = (SSLSocketFactory) SSLSocketFactory.getDefault();
        //System.out.println("Obtained the default SSL socket factory");      
    }
    
    private static boolean connectToRedMan1(String hostname) {
        try {           
            // Connect to RedMan
            trace.debug("Will try to connect to RedMan on host " + hostname 
                                                        + ", port = " + String.valueOf(rmport));
            SSLSocket so = (SSLSocket) sslFact.createSocket(hostname, rmport);
            //System.out.println("Connected to RedMan on host " + hostname);
            so.setTcpNoDelay(true);
            so.setSoTimeout(20000);
            
            // Get I/O streams to talk to RedMan                
            in1  = new BufferedReader(new InputStreamReader(so.getInputStream()));
            out1 = new PrintWriter(new OutputStreamWriter(so.getOutputStream()));
            
            return(true);
        }
        catch (ConnectException e) {
            System.out.println("RedMan (on " + hostname + ") not allowing connection: " + e.getMessage());
            return(false);
        }
        catch (SocketException e) {
            System.out.println("Could not connect to RedMan (on " + hostname + "): " + e.getMessage());
            return(false);
        }
        catch (Exception e) {
            e.printStackTrace();
            return(false);
        }
    }

    private static boolean connectToRedMan2(String hostname) {
        try {           
            // Connect to RedMan
            trace.debug("Will try to connect to RedMan on host " + hostname 
                                                        + ", port = " + String.valueOf(rmport));
            SSLSocket so = (SSLSocket) sslFact.createSocket(hostname, rmport);
            //System.out.println("Connected to RedMan on host " + hostname);
            so.setTcpNoDelay(true);
            so.setSoTimeout(20000);
            
            // Get I/O streams to talk to RedMan                
            in2  = new BufferedReader(new InputStreamReader(so.getInputStream()));
            out2 = new PrintWriter(new OutputStreamWriter(so.getOutputStream()));
            
            return(true);
        }
        catch (ConnectException e) {
            System.out.println("RedMan (" + hostname + ") not allowing connection: " + e.getMessage());
            return(false);
        }
        catch (SocketException e) {
            System.out.println("Could not connect to RedMan (" + hostname + "): " + e.getMessage());
            return(false);
        }
        catch (Exception e) {
            e.printStackTrace();
            return(false);
        }
    }
    
    private static boolean connectToRedMen() {
        if (!connectToRedMan1(host1)) return(false);
        if (!authenticateWithRedMan(host1, in1, out1)) {
            trace.error("Unable to authenticate with RedMan on node " + host1);
            return(false);
        }
        if (appconfig.equals("REDUNDANT")) {
            if (!connectToRedMan2(host2)) return(false);
            if (!authenticateWithRedMan(host2, in2, out2)) {
                trace.error("Unable to authenticate with RedMan on node " + host2);
                return(false);
            }
        }
        return(true);
    }
    
     private static boolean authenticateWithRedMan(String hostname, BufferedReader in, PrintWriter out) {
        try {              
            if (out != null) {
                String msg = "OPINT " + ruleofthumb;
                             
                // System.out.println("opint: user/passwd to RedMan = " + msg); 
                out.println(msg);
                out.flush();
                
            }
            else trace.warn("Not connected to RedMan on host " + hostname);
            String resp = in.readLine();
            if ((resp == null) || (resp.length() == 0) || !resp.equals("ACK")) {
                trace.error("Authentication with RedMan (on " + hostname + ") failed - resp = " + resp); 
                return(false);
            }
            else {
                // System.out.println("Authentication suceeded");
                return(true);
            }
        }
        catch (Exception e) {
            System.out.println("Exception authenticating with RedMan (on " + hostname 
                                                                    + "): " + e.getMessage());
            return(false);
        }
    }   
    
    private static String readTestLine() {
        String line;
        try { 
            line = testin.readLine();
        }   
        catch (Exception e) {
            System.out.println("Exception: " + e.getMessage());
            return(null);
        }           

        // System.out.println("Read line '" + line + "'");
        return(line);
    }
       
    private static String getHosts() {
        String hostname, token;
        
        // Read the designated primary host
        primaryhost = emsdynacfgtable.getProperty("PRIMARY");
        if (primaryhost == null) return("Missing parameter");
        primaryhost = primaryhost.trim();
        
        // Read the host names where the application is supposed to run
        apphosts = emsstacfgtable.getProperty("EMS_HOSTS");
        if (apphosts == null) return("Missing parameter");
        
        // Get localhost name
        try {
            localhost = InetAddress.getLocalHost().getHostName();
        }   
        catch (UnknownHostException e) {
            trace.error(e.getMessage());
            return("Error reading the localhost name");
        }      
        
        // Get application host names
        StringTokenizer st = new StringTokenizer(apphosts);
        host1 = st.nextToken();
        
        if (appconfig.equals("REDUNDANT")) {
            host2 = st.nextToken();
            if (host1.equals(localhost)) {
                remotehost = host2;
            }
            if (host2.equals(localhost)) {
                remotehost = host1;
            }
        }
        
        return("OK");
    }
    
    public static boolean getApplicationInfo() {
        String token;
        
        // Obtain the config file names for the application
        emscfgfiledir = rmcfgtable.getProperty("EMS_HOMEDIR") + "/conf";
        emsstacfgfilepath = emscfgfiledir + "/" + EMSSTACFGFILENAME;
        emsdynacfgfilepath = emscfgfiledir + "/" + EMSDYNACFGFILENAME;
        
        // Read the application configuration files
        if (!loadEMSStaticConfigFile()) return(false);
        if (!loadEMSDynamicConfigFile()) return(false);
        
        // Read the application configuration
        token = emsstacfgtable.getProperty("EMS_HA_MODEL");
        if (token == null) return(false);
        else token = token.trim();
        if (token.equals("1")) appconfig = "SIMPLEX";
        if (token.equals("1+1")) appconfig = "REDUNDANT";
           
        // Read the application shutdown time
        token = emsstacfgtable.getProperty("EMS_SHUTDOWN_TIME");
        if (token == null) return(false);
        else token = token.trim();
        shutdowntime = Integer.parseInt(token);
        
        // Read the application initialization time
        token = emsstacfgtable.getProperty("EMS_INIT_TIME");
        if (token == null) return(false);
        else token = token.trim();
        inittime = Integer.parseInt(token);

        // Read the application state change time
        token = emsstacfgtable.getProperty("EMS_STATE_CHANGE_TIME");
        if (token == null) return(false);
        else token = token.trim();
        statechgtime = Integer.parseInt(token);
        
        // Read application dynamic configuration data
        token = emsdynacfgtable.getProperty("FAILOVER_ENABLED");
        if (token == null) return(false);
        else token = token.trim();
        if (token.equals("TRUE")) failoverenabled = true;
        else failoverenabled = false;
        token = emsdynacfgtable.getProperty("START_ENABLED");
        if (token == null) return(false);
        else token = token.trim();
        if (token.equals("TRUE")) startenabled = true;
        else startenabled = false;
        
        // Read information about hosts
        if (!getHosts().equals("OK")) return(false);
        
        return(true);
    }
    
    private static boolean openTestConfigFile() {
        try {
            testin = new BufferedReader(new FileReader(testcfgfilepath));           
            return(true);
        }
        catch (FileNotFoundException e) {
            trace.error("Test config file not found");
            return(false);
        }        
        catch (IOException e) {
            trace.error("Unable to open config file " + testcfgfilepath 
                                                                  + ": IOException,  " + e.getMessage());
            return(false);
        }
    }

    private static boolean loadRMConfigFile() {
        try {
            rmcfgfile = new FileInputStream(rmcfgfilepath);           
        }
        catch (FileNotFoundException e) {
            trace.error("RedMan config file not found");
            return(false);
        }        
        try {
            rmcfgtable = new Properties();
            rmcfgtable.load(rmcfgfile);
            return(true);
        }
        catch (IOException e) {
            trace.error("Unable to open/read config file " + rmcfgfilepath 
                                                                  + ": IOException,  " + e.getMessage());
            return(false);
        }
    }
    
    private static boolean loadEMSStaticConfigFile() {
        try {
            emsstacfgfile = new FileInputStream(emsstacfgfilepath);           
        }
        catch (FileNotFoundException e) {
            trace.error("EMS static config file not found");
            return(false);
        }        
        try {
            emsstacfgtable = new Properties();
            emsstacfgtable.load(emsstacfgfile);
            return(true);
        }
        catch (IOException e) {
            trace.error("Unable to open/read config file " + emsstacfgfilepath 
                                                                  + ": IOException,  " + e.getMessage());
            return(false);
        }
    }
    
    private static boolean loadEMSDynamicConfigFile() {
        try {
            emsdynacfgfile = new FileInputStream(emsdynacfgfilepath);           
        }
        catch (FileNotFoundException e) {
            trace.error("EMS static config file not found");
            return(false);
        }        
        try {
            emsdynacfgtable = new Properties();
            emsdynacfgtable.load(emsdynacfgfile);
            return(true);
        }
        catch (IOException e) {
            trace.error("Unable to open/read config file " + emsdynacfgfilepath 
                                                                  + ": IOException,  " + e.getMessage());
            return(false);
        }
    }
    
    private static boolean getInitialTestData() {
        // Read the RedMan configuration file
        if (!loadRMConfigFile()) return(false);
        
        // Read the RedMan port
        String token = rmcfgtable.getProperty("REDMAN_PORT");
        if (token == null) return(false);
        token = token.trim();
        rmport = Integer.parseInt(token);
        
        // Open the Test configuration file
        if (!openTestConfigFile()) return(false);
        
        // Read the EMS config file directory
        if (!getApplicationInfo()) return(false);
        return(true);
    }
    
    private static boolean handleParameters(String args[]) {
        // Check which test to start with
        if ((args.length > 0) && (args[0].startsWith("-t"))) {
            fromstart = false;
            firsttest = Integer.parseInt(args[0].substring(2));
        }       

        if (args.length > 2) {
            System.out.println("USAGE: java <package-name>.Test [-t<n>] <rm-config-file-dir>");
            return(false);
        }        
        if (args.length == 0) {
            rmcfgfilepath = RMCFGFILENAME;
        }
        if (args.length == 1) {
            if (!args[0].startsWith("-t")) {
                rmcfgfiledir = args[0];
                rmcfgfilepath = rmcfgfiledir + "/" + RMCFGFILENAME;
            }
            else {
                rmcfgfilepath = RMCFGFILENAME;
            }
        }         
        if (args.length == 2) {
            if (!args[0].startsWith("-t")) {
                System.out.println("USAGE: java <package-name>.Test [-t<n>] <rm-config-file-dir>");
                return(false);
            }
            rmcfgfiledir = args[1];
            rmcfgfilepath = rmcfgfiledir + "/" + RMCFGFILENAME;
        }
        
        // Initialize log4j
        logstart = new Log4Test();
        trace = Category.getInstance(Test.class);
               
        return(true);
        
    }
    
    public static void main (String args[]) {
        if (!handleParameters(args)) {
            System.out.println("Invalid test program parameters. Exiting ...");
            System.exit(1);
        }
        
        trace.info("\n\n--------------------------------- EMS INTEGRATION TEST PROGRAM "
                                                   + "-----------------------------------\n");
        if (!getInitialTestData()) {
            trace.error("Could not get test data. Exiting ...");
            System.exit(1);
        }
       
        // Initialize SSL
        initSSL();
        
        // Connect to both RedMan        
        if (connectToRedMen()) {
            boolean res = initializeTestOutLog();
            if (!res) return;
            inhibited = !failoverenabled;
            do {
                line = readTestLine(); 
                if (line == null) break;
                line = line.trim();
                // trace.info("line = '" + line + "'");
                if ((line.length() == 0) || (line.charAt(0) == ' ') 
                                         || (line.charAt(0) == '#')) continue;               
                processCmd(line);
            } while (!stop);
            if (!stop) {
                trace.info("\n\n                             ***  FINAL TEST RESULT = SUCCESS  *** "
                                                   + "\n\n----------------------"
                                                   + "--------------------------------------"
                                                   + "--------------------------------------\n");
            }
            else {
                trace.info("\n\n                             ***  FINAL TEST RESULT = FAILURE  *** "
                                                   + "\n\n----------------------"
                                                   + "--------------------------------------"
                                                   + "--------------------------------------\n");
            }
            closeTestOutLog();
        }
        else trace.error("Unable to connect to RedMen");
    }
    
    static String line, cmd, primary, state1, state2, host, otherhost, procname;
    static String targetstate1, targetstate2, targetprimary;
    static PrintWriter logout;
    static File logfile;
    static boolean inhibited, targetinhibited, backup, both = false;
    
    private static void processCmd(String line) {
        if (parseCmd(line)) execCmd();
        else {
            logout.println(line + " - FAILED (invalid command)\n");
            logout.flush();
            stop = true;
            trace.info("Test '" + line + "' - FAILED (invalid command)\n"); 
        }
    }
    
    private static void execCmd() {
        if (cmd.equals("SWITCH")) execSwitchTest();
        if (cmd.equals("REBOOT")) execRebootTest();
        if (cmd.equals("INHIBIT")) execInhibitTest();
        if (cmd.equals("UNINHIBIT")) execUninhibitTest();
        if (cmd.startsWith("PRIMARY")) execPrimaryTest();
        if (cmd.equals("TEST")) execTestTest();
        if (cmd.equals("UNTEST")) execUntestTest();
        if (cmd.equals("RESET")) execResetTest();
        if (cmd.equals("CHECK")) execCheckTest();
        if (cmd.equals("KILLPROC")) execKillProcTest();
        if (cmd.equals("KILLRM")) execKillRMTest();
    }
    
    private static boolean parseCmd(String line) {
        String token;
        StringTokenizer st = new StringTokenizer(line);
        cmd = st.nextToken();
        
        // System.out.println("Command: '" + cmd + "'");
        
        if (!cmd.equals("START")      &&   !cmd.equals("SHUTDOWN")    &&
            !cmd.equals("ENABLE")     &&   !cmd.equals("DISABLE")     &&
            !cmd.equals("INHIBIT")    &&   !cmd.equals("UNINHIBIT")   &&
            !cmd.equals("SWITCH")     &&   !cmd.equals("RESTART")     &&
            !cmd.equals("TEST")       &&   !cmd.equals("UNTEST")      &&
            !cmd.equals("ACTIVE")     &&   !cmd.equals("STANDBY")     &&
            !cmd.equals("QUERY")      &&   !cmd.equals("RELOAD")      &&
            !cmd.equals("ACTIVE")     &&   !cmd.equals("STANDBY")     &&
            !cmd.equals("SHUTDOWN-F") &&   !cmd.equals("DISABLE-F")   &&
            !cmd.equals("RESTART-F")  &&   !cmd.equals("PRIMARY")     &&
            !cmd.equals("REBOOT")     &&   !cmd.equals("CHECK")       &&
            !cmd.equals("RESET")      &&   !cmd.equals("KILLPROC")    &&
                                           !cmd.equals("KILLRM"))        {
                return(false);
        } 
        
        if (cmd.equals("PRIMARY") || cmd.equals("REBOOT") || cmd.equals("RESET")
                                  || cmd.equals("UNTEST") || cmd.equals("TEST")) {
            if (!st.hasMoreTokens()) return(false);
            host = st.nextToken();
            if (cmd.equals("REBOOT") && st.hasMoreTokens()) {
                otherhost = st.nextToken();
                both = true;
            }
            if (cmd.equals("PRIMARY")) cmd += " " + host;
        }
        
        if (cmd.equals("CHECK")) {
            if (st.countTokens() < 4) return(false);
            targetstate1 = st.nextToken();
            targetstate2 = st.nextToken();
            targetprimary = st.nextToken();
            token = st.nextToken();
            if (token.equals("INHIBITED")) targetinhibited = true;
            else targetinhibited = false;
        }
        
        if (cmd.equals("KILLRM")) {
            if (st.hasMoreTokens()) {
                token = st.nextToken();
                if (token.equals("BACKUP")) {
                    backup = true;
                    if (st.hasMoreTokens()) host = st.nextToken();
                    else {
                        trace.error("Test command incomplete");
                        return(false);
                    }
                }
                else {
                    host = token;
                    backup = false;
                }
            }
            else return(false);
        }
        
        if (cmd.equals("KILLPROC")) {
            numproc = 0;
            if (st.hasMoreTokens()) {
                procname = st.nextToken();
                if (st.hasMoreTokens()) {
                    host = st.nextToken();
                    if (st.hasMoreTokens()) {
                        token = st.nextToken();
                        if (token.equals("EQUAL") || token.equals("DIFF")) {
                            equality = token;
                            int i;
                            if ((i=st.countTokens()) > 0) {
                                // Read proc names
                                numproc = i;
                                procnames = new String[numproc];
                                procpids = new String[numproc];
                                for (i = 0; i < numproc; i++) procnames[i] = st.nextToken();                                   
                            }
                        }
                    }
                }
                else {
                    trace.error("Test command incomplete");
                    return(false);
                }
            }
            else return(false);
        }
           
        return(true);
    }
    
    private static void execCheckTest() {
        boolean res;
        
        // Query to get the state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("CURRENT STATE: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
        
        // Compare results
        boolean success = false;
            
        if (state1.equals(targetstate1) && state2.equals(targetstate2) 
              && (inhibited == targetinhibited) && primary.equals(targetprimary)) 
            success = true;
  
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            trace.info("Test '" + line + "' - FAILED\n");           
        }      
    }
    
    
    private static void execResetTest() {
        boolean res, oldinhibited;
        String resp, oldstate1, oldstate2, oldprimary;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1; 
        oldprimary = primary;
        oldinhibited = inhibited;
        oldstate2 = state2;
        trace.debug("BEFORE RESET (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited); 
        
        // Send primary command to RedMan
        res = sendCmdToRedMan(host1, appname, "PRIMARY " + host, out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the primary command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Primary sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED");
            logout.flush();
            stop = true;
            return;
        }
        
        // Send uninhibit command to RedMan
        res = sendCmdToRedMan(host1, appname, "UNINHIBIT", out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the uninhibit command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Uninhibit sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            return;
        }
        
        // Send commands to restart the application on both hosts
        res = sendCmdToRedMan(host1, appname, "RESTART-F", out1);
        if (!res) {
            stop = true;
            return;
        }
        res = sendCmdToRedMan(host2, appname, "RESTART-F", out2);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait
        try {
            Thread.sleep(shutdowntime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from both RedMen for the restart command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Restart sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            return;
        }
        resp = receiveResponse(host2, in2);
        if (!resp.startsWith("ACK")) {
            trace.error("Restart sent to RedMan on host " + host2 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            return;
        }
        
        // Wait for the two applications to reinitialize
        try {
            Thread.sleep(inittime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER RESET (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        if (host.equals(host1)) {
            if (state1.equals("ACTIVE") && state2.equals("STANDBY") 
                           && !inhibited && primary.equals(host1)) success = true;
        }
        
        if (host.equals(host2)) {
            if (state2.equals("ACTIVE") && state1.equals("STANDBY") 
                           && !inhibited && primary.equals(host2)) success = true;
        }

        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            trace.info("Test '" + line + "' - FAILED\n");           
        }         
    }
    
    private static void execPrimaryTest() {
        boolean res, oldinhibited;
        String resp, oldstate1, oldstate2, oldprimary;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1; 
        oldprimary = primary;
        oldinhibited = inhibited;
        oldstate2 = state2;
        trace.debug("BEFORE PRIMARY (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited); 
        
        // Send command to RedMan
        res = sendCmdToRedMan(host1, appname, cmd , out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the primary command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Primary sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED");
            logout.flush();
            stop = true;
            return;
        }
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER PRIMARY (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        if (oldstate1.equals(state1) && oldstate2.equals(state2) 
                   && (oldinhibited == inhibited) && primary.equals(host)) success = true;

        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            trace.info("Test '" + line + "' - FAILED\n");           
        }                
    }
    
    private static void execUntestTest() {
        boolean res;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        trace.debug("BEFORE UNTEST (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);

        if (host.equals(host1)) {
            // Send command to RedMan
            res = sendCmdToRedMan(host1, appname, cmd, out1);
            if (!res) {
                stop = true;
                return;
            }
            
            // Receive response from RedMan for the untest command
            resp = receiveResponse(host1, in1);
            if (!resp.startsWith("ACK")) {
                if (oldstate1.equals("MAINTENANCE")) {
                    trace.error("Untest sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
                    logout.println(line + " - FAILED\n");
                    stop = true;
                }
                else {
                    trace.info("Test '" + line + "' - PASSED\n");
                    logout.println(line + " - PASSED\n");
                }
                logout.flush();
                return;
            }
        }
            
        if (host.equals(host2)) {
            // Send command to RedMan
            res = sendCmdToRedMan(host2, appname, cmd, out2);
            if (!res) {
                stop = true;
                return;
            }
            
            // Receive response from RedMan for the untest command
            resp = receiveResponse(host2, in2);
            if (!resp.startsWith("ACK")) {
                if (oldstate2.equals("MAINTENANCE")) {
                    trace.error("Untest sent to RedMan on host " + host2 + " FAILED - Resp = " + resp);
                    logout.println(line + " - FAILED");
                    stop = true;
                }
                else {
                    trace.info("Test '" + line + "' - PASSED\n");
                    logout.println(line + " - PASSED\n");                    
                }
                logout.flush();
                return;
            }
        }
        
        // Wait 
        try {
            Thread.sleep(statechgtime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
                    
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER UNTEST (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;

        if (host.equals(host1)) {
            if (oldstate1.equals("MAINTENANCE") && state1.equals("STANDBY") && !inhibited 
                && primary.equals(host2) && oldstate2.equals("STANDBY") && state2.equals("ACTIVE")) 
                success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("ACTIVE") && !inhibited 
                && primary.equals(host1) && oldstate2.equals("STANDBY") && state2.equals("STANDBY")) 
                success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("ACTIVE") && !inhibited 
                && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) 
                success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("STANDBY") && !inhibited
                && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) 
                success = true;
        }
        if (host.equals(host2)) {
            if (oldstate2.equals("MAINTENANCE") && state2.equals("STANDBY") && !inhibited 
                && primary.equals(host1) && oldstate1.equals("STANDBY") && state1.equals("ACTIVE")) 
                success = true;
            if (oldstate2.equals("MAINTENANCE") && state2.equals("ACTIVE") && !inhibited 
                && primary.equals(host2) && oldstate1.equals("STANDBY") && state1.equals("STANDBY")) 
                success = true;
            if (oldstate2.equals("MAINTENANCE") && state2.equals("ACTIVE") && !inhibited 
                && oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE")) 
                success = true;
            if (oldstate2.equals("MAINTENANCE") && state2.equals("STANDBY") && !inhibited
                && oldstate1.equals("ACTIVE") && state1.equals("ACTIVE")) 
                success = true;
        }
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");    
            stop = true;
        }                
    }
    
    private static void execTestTest() {
        boolean res;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        trace.debug("BEFORE TEST (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);

        if (host.equals(host1)) {
            // Send command to RedMan
            res = sendCmdToRedMan(host1, appname, cmd, out1);
            if (!res) {
                stop = true;
                return;
            }
            
            // Receive response from RedMan for the test command
            resp = receiveResponse(host1, in1);
            if (!resp.startsWith("ACK")) {
                if (oldstate1.equals("STANDBY")) {
                    trace.error("Test sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
                    logout.println(line + " - FAILED\n");
                    stop = true;
                }
                else {
                    trace.info("Test '" + line + "' - PASSED\n");
                    logout.println(line + " - PASSED\n");
                }
                logout.flush();
                return;
            }
        }
            
        if (host.equals(host2)) {
            // Send command to RedMan
            res = sendCmdToRedMan(host2, appname, cmd, out2);
            if (!res) {
                stop = true;
                return;
            }
            
            // Receive response from RedMan for the test command
            resp = receiveResponse(host2, in2);
            if (!resp.startsWith("ACK")) {
                if (oldstate2.equals("STANDBY")) {
                    trace.error("Test sent to RedMan on host " + host2 + " FAILED - Resp = " + resp);
                    logout.println(line + " - FAILED");
                }
                else {
                    trace.info("Test '" + line + "' - PASSED\n");
                    logout.println(line + " - PASSED\n");                    
                }
                logout.flush();
                return;
            }
        }
        
        // Wait 
        try {
            Thread.sleep(statechgtime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
         
        // Query to get final state
        if (host.equals(host1)) res = queryRedMan(host1);
        if (host.equals(host2)) res = queryRedMan(host2);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER TEST (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;

        if (host.equals(host1)) {
            if (oldstate1.equals("STANDBY") && state1.equals("MAINTENANCE") 
                                  && inhibited && oldstate2.equals(state2)) success = true;
        }
        if (host.equals(host2)) {
            if (oldstate2.equals("STANDBY") && state2.equals("MAINTENANCE")
                                  && inhibited && oldstate1.equals(state1)) success = true;
        }
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");    
            stop = true;
        }        
    }
    
    private static void execUninhibitTest() {
        boolean res;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        trace.debug("BEFORE UNINHIBIT: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);     
        // Send command to RedMan
        res = sendCmdToRedMan(host1, appname, cmd, out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the uninhibit command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Uninhibit sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            return;
        }
        
        // Query to get final state
        if (host.equals(host1)) res = queryRedMan(host1);
        if (host.equals(host2)) res = queryRedMan(host2);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER UNINHIBIT: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
        
        // Compare results
        boolean success = false;
        if (oldstate1.equals(state1) && oldstate2.equals(state2) && !inhibited) success = true;
        
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            stop = true;
        }                
    } 
    
    private static void execInhibitTest() {
        boolean res;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1; 
        oldstate2 = state2;
        trace.debug("BEFORE INHIBIT: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);     
        // Send command to RedMan
        res = sendCmdToRedMan(host1, appname, cmd, out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the inhibit command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            trace.error("Inhibit sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
            logout.println(line + " - FAILED\n");
            logout.flush();
            stop = true;
            return;
        }
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER INHIBIT: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        if (oldstate1.equals(state1) && oldstate2.equals(state2) && inhibited) success = true;

        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            stop = true;
        }        
    }
    
    private static void execKillRMTest() {
        boolean res, oldinhibited;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            if (backup) backup = false;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        oldinhibited = inhibited;
        trace.debug("BEFORE KILLRM (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
        
        // Kill the requested RedMan
        String strmpid = getPidFromRedMan(host, "REDMAN", backup);
        if (strmpid == null) {
            stop = true;
            if (backup) backup = false;
            return;
        }
        res = killProc(host, strmpid);
        if (!res) {
            stop = true;
            if (backup) backup = false;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(15000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        if (!backup) {
            if (host.equals(host1)) {
                if (!connectToRedMan1(host1) || !authenticateWithRedMan(host1, in1, out1)) {
                    logout.println(line + " - FAILED\n");
                    logout.flush();
                    trace.info("Test '" + line + "' - FAILED\n");
                    stop = true;
                    if (backup) backup = false;
                    return;
                }
            }
            
            if (host.equals(host2)) {
                if (!appconfig.equals("REDUNDANT") || !connectToRedMan2(host2) 
                                                   || !authenticateWithRedMan(host2, in2, out2)) {
                    logout.println(line + " - FAILED\n");
                    logout.flush();
                    trace.info("Test '" + line + "' - FAILED\n");
                    stop = true;
                    if (backup) backup = false;
                    return;
                }
            }
        }
            
        // Query to get final state
        if (host.equals(host1)) res = queryRedMan(host1);
        if (host.equals(host2)) res = queryRedMan(host2);
        if (!res) {
            stop = true;
            if (backup) backup = false;
            return;
        }
        
        trace.debug("AFTER KILLRM (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        
        if (oldstate1.equals(state1) && oldstate2.equals(state2) && (oldinhibited == inhibited)) 
            success = true;
        else success = false;
        
        if (success) {
            String newpid = getPidFromRedMan(host, "REDMAN", backup);
            String name = "REDMAN";
            if (backup) name += "-BKP";
            if (newpid == null) {
                trace.error("Error: new pid of killed process " + name + "(" + host + ") is null");
                success = false;
            }
            else if (newpid.equals(strmpid)) {
                    trace.error("Error: new pid of killed process " + name + "(" + host + ") "
                                                        + "is equal to the old pid " + strmpid);
                    success = false;
            }
        }
        
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            stop = true;
        }
        
        if (backup) backup = false;
    }

    private static void execKillProcTest() {
        boolean res, oldinhibited;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        oldinhibited = inhibited;
        trace.debug("BEFORE KILLPROC " + procname + " (" + host + "): " + host1 + ":" + state1 
                                       + ", " + host2 + ":" + state2 + ", PRIMARY = " + primary 
                                       + ", INHIBITED:" + inhibited);
        
        // Get pids of processes to be checked
        if (numproc > 0) {
            for (int i=0; i < numproc; i++) {
                procpids[i] = getPidFromRedMan(host, procnames[i], false);
                if (procpids[i] == null) {
                    stop = true;
                    return;
                }
            }
        }
        
        // Kill the requested process
        String stprocpid = getPidFromRedMan(host, procname, false);
        if (stprocpid == null) {
            stop = true;
            return;
        }

        res = killProc(host, stprocpid);
        if (!res) {
            stop = true;
            return;
        }
        
        boolean critical = procCritical(procname);
        int sleeptime;
        if (critical) sleeptime = inittime + 20000;
        else sleeptime = 10000;
        trace.debug("Will sleep " + String.valueOf(sleeptime) + " milliseconds");
        
        // Wait 
        try {
            Thread.sleep(sleeptime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
          
        trace.debug("Woke up from sleep");
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER KILLPROC " + procname + " (" + host + "): " + host1 + ":" + state1 
                                      + ", " + host2 + ":" + state2  + ", PRIMARY = " + primary 
                                      + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;

        if (!critical) {
            if (oldstate1.equals(state1) && oldstate2.equals(state2) && (oldinhibited == inhibited)) 
                success = true;
            else success = false;
        }
        
        if (critical && host.equals(host1)) {
            if (oldstate1.equals("STANDBY") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("STANDBY") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
        }
        if (critical && host.equals(host2)) {
            if (oldstate1.equals("STANDBY") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("STANDBY") && state1.equals("STANDBY") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENACE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
        }
        
        if (success) {
            String newpid = getPidFromRedMan(host, procname, false);
            if (newpid == null) {
                trace.error("Error: new pid of killed process " + procname + "(" + host + ") is null");
                success = false;
            }
            else if (newpid.equals(stprocpid)) {
                    trace.error("Error: new pid of killed process " + procname + "(" + host + ") "
                                                        + "is equal to the old pid " + stprocpid);
                    success = false;
            }
        }
        
        // Check pids of checked processes
        String stpid;
        if (numproc > 0) {
            for (int i=0; i < numproc; i++) {
                stpid = getPidFromRedMan(host, procnames[i], false);
                if (stpid == null) {                   
                    stop = true;
                    return;
                }
                if (equality.equals("EQUAL") && !stpid.equals(procpids[i])) {
                    trace.debug("Error: pids of process " + procnames[i] 
                                    + "(" + procpids[i] + ", " + stpid + ") should be equal");                    
                    stop = true;
                    return;
                }
                if (equality.equals("DIFF" ) && stpid.equals(procpids[i])) {
                    trace.debug("Error: pids of process " + procnames[i] 
                                    + "(" + procpids[i] + ", " + stpid + ") should be different");                    
                    stop = true;
                    return;                    
                }
            }
        }
        
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            stop = true;
        }
    }
    
    private static boolean procCritical(String pname) {
        // Read the fault threshold of the process
        String stthreshold = emsstacfgtable.getProperty("EMS_" + procname + "_FAULT_THRESHOLD");
        if (stthreshold == null) return(false);
        stthreshold = stthreshold.trim();
        if (stthreshold.equals("0")) return(true);
        else return(false);
    }
    
    private static void execRebootTest() {
        boolean res, oldinhibited;
        String resp, oldstate1, oldstate2, oldprimary;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            both = false;
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        oldprimary = primary;
        oldinhibited = inhibited;
        trace.debug("BEFORE REBOOT (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
        
        // Reboot the requested host
        res = rebootHost(host);
        if (!res) {
            both = false;
            stop = true;
            return;
        }
        
        // Check if should reboot second host
        if (both) {
            if (!appconfig.equals("REDUNDANT")) {
                both = false;
                stop = true;
                return;
            }
            res = rebootHost(otherhost);
            if (!res) {
                both = false;
                stop = true;
                return;
            }
        }
        
        // Wait 
        try {
            Thread.sleep(330000);   // 5 minutes : 30 seconds
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        boolean testres = false;
        
        if (host.equals(host1)) {
            testres = (connectToRedMan1(host1) && authenticateWithRedMan(host1, in1, out1));
            if (!testres) {
                logout.println(line + " - FAILED\n");
                logout.flush();
                trace.info("Test '" + line + "' - FAILED\n");
                both = false;
                stop = true;
                return;
            }
        }
            
        if (host.equals(host2)) {
            testres = (connectToRedMan2(host2) && authenticateWithRedMan(host2, in2, out2));
            if (!testres) {
                logout.println(line + " - FAILED\n");
                logout.flush();
                trace.info("Test '" + line + "' - FAILED\n");
                both = false;
                stop = true;
                return;
            }
        }
        
        if (both) {            
            if (host.equals(host1)) 
                testres = (connectToRedMan2(host2) && authenticateWithRedMan(host2, in2, out2));
            else
                testres = (connectToRedMan1(host1) && authenticateWithRedMan(host1, in1, out1));
            if (!testres) {
                logout.println(line + " - FAILED\n");
                logout.flush();
                trace.info("Test '" + line + "' - FAILED\n");
                both = false;
                stop = true;
                return;
            }
        }
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            both = false;
            stop = true;
            return;
        }
        trace.debug("AFTER REBOOT (" + host + "): " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        
        if (!both && host.equals(host1)) {
            if (oldstate1.equals("STANDBY") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("STANDBY") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("STANDBY") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
        }
        if (!both && host.equals(host2)) {
            if (oldstate1.equals("STANDBY") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("STANDBY") && state1.equals("STANDBY") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && inhibited
                    && oldstate2.equals("STANDBY") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("STANDBY")) success = true;
            if (oldstate1.equals("ACTIVE") && state1.equals("ACTIVE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && !inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("ACTIVE") && state2.equals("MAINTENANCE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENACE") && !inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("ACTIVE")) success = true;
            if (oldstate1.equals("MAINTENANCE") && state1.equals("MAINTENANCE") && inhibited
                    && oldstate2.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
        }
        
        if (both) {
            if (!oldinhibited) {
                if (oldprimary.equals(host1) && state1.equals("ACTIVE") 
                                             && state2.equals("STANDBY")) success = true;
                if (oldprimary.equals(host2) && state1.equals("STANDBY")
                                             && state2.equals("ACTIVE"))  success = true;
            }
            else if (state1.equals("MAINTENANCE") && state2.equals("MAINTENANCE")) success = true;
            both = false;
        }   
        
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            both = false;
            stop = true;
        }
    }

    private static boolean killProc(String hname, String ppid) {       
        String cmd = "rsh " + hname + " kill -9 " + ppid;
        try {
            trace.debug("Will execute: " + cmd);
            Runtime.getRuntime().exec(cmd);
            trace.debug("Executed: " + cmd);
        }
        catch (IOException e) {
            trace.error("Runtime.exec() failed - " + e.getMessage());
            return(false);
        }
        return(true);
    }    
    
    private static boolean rebootHost(String hostname) {
        String cmd = "rsh " + hostname + " shutdown -y -g0 -i6";
        try {
            trace.debug("Executing: " + cmd);
            Runtime.getRuntime().exec(cmd);
        }
        catch (IOException e) {
            trace.error("Runtime.exec() failed - " + e.getMessage());
            return(false);
        }
        return(true);
    }
    
    private static void execSwitchTest() {
        boolean res;
        String resp, oldstate1, oldstate2;
        
        // Query to get initial state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        oldstate1 = state1;
        oldstate2 = state2;
        trace.debug("BEFORE SWITCH: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);     
        // Send command to RedMan
        res = sendCmdToRedMan(host1, appname, cmd, out1);
        if (!res) {
            stop = true;
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(1000);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Receive response from RedMan for the switch command
        resp = receiveResponse(host1, in1);
        if (!resp.startsWith("ACK")) {
            if (inhibited || oldstate1.equals("MAINTENANCE") || oldstate2.equals("MAINTENANCE")) {
                logout.println(line + " - PASSED\n");
                trace.info("Test '" + line + "' - PASSED\n");
            }
            else {
                trace.error("Switch sent to RedMan on host " + host1 + " FAILED - Resp = " + resp);
                logout.println(line + " - FAILED\n");
                stop = true;
            }
            logout.flush();
            return;
        }
        
        // Wait 
        try {
            Thread.sleep(statechgtime);
        }
        catch (Exception e) {
            trace.warn("Test program had interrupted sleep");
        }
        
        // Query to get final state
        res = queryRedMan(host1);
        if (!res) {
            stop = true;
            return;
        }
        trace.debug("AFTER SWITCH: " + host1 + ":" + state1 + ", " + host2 + ":" + state2 
                                     + ", PRIMARY = " + primary + ", INHIBITED:" + inhibited);
                
        // Compare results
        boolean success = false;
        if (oldstate1.equals("ACTIVE") && state1.equals("STANDBY") 
                    && oldstate2.equals("STANDBY") && state2.equals("ACTIVE")) success = true;
        if (oldstate2.equals("ACTIVE") && state2.equals("STANDBY") 
                    && oldstate1.equals("STANDBY") && state1.equals("ACTIVE")) success = true;
    
        if (success) {
            logout.println(line + " - PASSED\n");
            logout.flush();
            trace.info("Test '" + line + "' - PASSED\n");
        }
        else {
            logout.println(line + " - FAILED\n");
            logout.flush();
            trace.info("Test '" + line + "' - FAILED\n");
            stop = true;
        }
    }
    
    private static boolean sendCmdToRedMan(String host, String app, String cmd, PrintWriter out) {
        try {              
            if (out != null) {
                String msg = "OPINT " + app + " " + cmd;
                             
                trace.debug("Command sent to RedMan = " + msg); 
                out.println(msg);
                out.flush();
                return(true);
            }
            else trace.info("sendCommandToRedMan(" + cmd + "): not connected to RedMan");
            return(false);
        }
        catch (Exception e) {
            trace.error("Exception (cmd = " + cmd + ", host = " + host + "): " + e.getMessage());
            return(false);
        }
    }

    private static boolean queryRedMan(String host) {
        BufferedReader in = null;
        PrintWriter out = null;
        
        if (host.equals(host1)) {
            in = in1;
            out = out1;
        }
        if (host.equals(host2)) {
            in = in2;
            out = out2;
        }
            
        try {
            out.println("OPINT EMS QUERY");
            out.flush();
            
            String resp = readQueryResponse(host, in);
            if (!resp.startsWith("ACK")) {
                trace.error("Query to RedMan on host " + host + " FAILED - Resp = " + resp);
                logout.println(line + " - FAILED");
                logout.flush();
                return(false);
            }
            else {
                parseQueryResponse(resp);
                return(true);
            }
        }
        catch (Exception e) {
            trace.error("Exception querying RedMan on host " + host + ": " + e.getMessage());
            return(false);
        }
    }
    
    private static String getPidFromRedMan(String hname, String pname, boolean backup) {
        BufferedReader in = null;
        PrintWriter out = null;
        
        if (hname.equals(host1)) {
            in = in1;
            out = out1;
        }
        if (hname.equals(host2)) {
            in = in2;
            out = out2;
        }
            
        String cmd = "OPINT ";
        if (pname.equals("REDMAN")) {
            cmd += "REDMAN GETPID ";
            if (backup) cmd += "BACKUP ";
        }
        else cmd += "EMS GETPID " + pname;
        
        trace.debug("GetPid command to RedMan (" + hname + "): '" + cmd + "'");
        
        try {
            out.println(cmd);
            out.flush();
            
            String stprocpid = readProcPid(hname, in);
            if (stprocpid == null) {
                String name;
                if (pname.equals("REDMAN") && backup) name = "REDMAN-BKP";
                else name = pname;
                trace.error("GetPid of process " + pname + " from RedMan on host " 
                                                 + name + " FAILED");
                logout.println(line + " - FAILED");
                logout.flush();
                return(null);
            }
            else {
                try {
                    if (Integer.parseInt(stprocpid) <= 0) {
                        return(null);
                    }
                }
                catch (Exception e) {
                    trace.debug("Got exception translating pid from application process - " + e.getMessage());
                    return(null);
                }
                return(stprocpid);
            }
        }
        catch (Exception e) {
            String name;
            if (pname.equals("REDMAN") && backup) name = "REDMAN-BKP";
            else name = pname;
            trace.error("Exception getting pid of process " + pname + " from RedMan on host " 
                                                + hname + ": " + e.getMessage());
            logout.println(line + " - FAILED");
            logout.flush();
            return(null);
        }
    }
    
    
    private static void parseQueryResponse(String resp) {
        String value;
        int pos;
        
        // Get the initial position for host1
        pos = resp.indexOf(host1);
        
        // Get the first collon after host1
        pos = resp.indexOf(':', pos+1);
        
        // Get state of host1
        value = resp.substring(pos+1);
        if (value.startsWith("ACTIVE")) state1 = "ACTIVE";
        if (value.startsWith("STANDBY")) state1 = "STANDBY";
        if (value.startsWith("FAILED")) state1 = "FAILED";
        if (value.startsWith("INIT")) state1 = "INIT";
        if (value.startsWith("OFFLINE")) state1 = "OFFLINE";
        if (value.startsWith("UNKNOWN")) state1 = "UNKNOWN";
        if (value.startsWith("MAINTENANCE")) state1 = "MAINTENANCE";
        
        // Get the second collon after host1
        pos = resp.indexOf(':', pos+1);
        
        // Get whether host1 is primary or secondary
        value = resp.substring(pos+1);
        if (value.startsWith("PRIMARY")) primary = host1;
        
        // Get the initial position for host2
        pos = resp.indexOf(host2);
        
        // Get the first collon after host2
        pos = resp.indexOf(':', pos+1);
        
        // Get state of host2
        value = resp.substring(pos+1);
        if (value.startsWith("ACTIVE")) state2 = "ACTIVE";
        if (value.startsWith("STANDBY")) state2 = "STANDBY";
        if (value.startsWith("FAILED")) state2 = "FAILED";
        if (value.startsWith("OFFLINE")) state2 = "OFFLINE";
        if (value.startsWith("MAINTENANCE")) state2 = "MAINTENANCE";
        
        // Get the second collon after host2
        pos = resp.indexOf(':', pos+1);
        
        // Get whether host2 is primary or secondary
        value = resp.substring(pos+1);
        if (value.startsWith("PRIMARY")) primary = host2;
        
        // Get the inhibited value
        pos = resp.indexOf("UNINHIBITED");
        if (pos == -1) inhibited = true;
        else inhibited = false;
    }
    
    private static String receiveResponse(String hostname, BufferedReader in) {
        try {
            String resp = in.readLine();
            if (resp == null) resp = "Null response received from RedMan";
            else if (resp.length() == 0) resp = "Zero-length response received from RedMan";
            return(resp);
        }
        catch (InterruptedIOException e) {
            String errorMsg = "Request to RedMan (on " + hostname + ") timed out";
            trace.error(errorMsg);
            return(errorMsg);
        }
        catch (Exception e) {
            String errorMsg = e.getMessage();
            trace.error(errorMsg);
            return(errorMsg);
        }
    }
    
    private static String readQueryResponse(String hostname, BufferedReader in) {
        try {
            String resp = in.readLine();
            if (resp == null) resp = "Null response received from query to RedMan";
            else if (resp.length() == 0) resp = "Zero-length response received from query to RedMan";
            resp = "ACK = " + resp;
            return(resp);
        }
        catch (InterruptedIOException e) {
            String errorMsg = "Query to RedMan (on " + hostname + ") timed out";
            trace.error(errorMsg);
            return(errorMsg);
        }
        catch (Exception e) {
            String errorMsg = e.getMessage();
            trace.error(errorMsg);
            return(errorMsg);
        }
    }
    
    private static String readProcPid(String hostname, BufferedReader in) {
        String errorMsg = null;
        try {
            String resp = in.readLine();
            if (resp == null) errorMsg = "Null response received from GetPid to RedMan";
            else if (resp.length() == 0) errorMsg = "Zero-length response received from GetPid to RedMan";
            if (errorMsg != null) {
                trace.error(errorMsg);
                return(null);
            }
            trace.debug("Response from GetPid cmd sent to RedMan (" + hostname + "): '" + resp + "'");
            StringTokenizer st = new StringTokenizer(resp);
            String stpid = null;
            while (st.hasMoreTokens()) stpid = st.nextToken();
            trace.debug("Pid being returned as response to GetPid cmd: '" + stpid + "'");
            return(stpid);
        }
        catch (InterruptedIOException e) {
            errorMsg = "GetPid sent to RedMan (on " + hostname + ") timed out";
            trace.error(errorMsg + " - " + e.getMessage());
            return(null);
        }
        catch (Exception e) {
            errorMsg = e.getMessage();
            trace.error(errorMsg);
            return(null);
        }
    }
    
    static private boolean initializeTestOutLog() {
        try {
            logfile = new File(".", "TestOut.log");
            if (logfile.exists()) logfile.delete();
        
            logfile.createNewFile();
        
            logout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(logfile))); 
        
            logout.println("\n\n----------------------------------------------------------------\n");
            logout.println("                          TEST RESULTS               \n");
            logout.flush();
            return(true);
        }
        catch (Exception e) {
            trace.error("Could not open file TestOut.log");
            return(false);
        }
    }
    
    static private void closeTestOutLog() {
        logout.println("\n----------------------------------------------------------------\n");
        logout.flush();
        logout.close();
    }
}
